﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Data.SqlClient;
using System.Configuration;
using System.Text.RegularExpressions;
using CommonLib;
using System.Threading;
using System.IO;

namespace pageAnalyzer
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
            connString = ConfigurationManager.ConnectionStrings["pageAnalyzer.Properties.Settings.CrawlPagesConnectionString"].ToString();
            contentConn = new SqlConnection(connString);
        }

        private  Dictionary<string, DomainConfigurationElement> GetDomainConfigurationDict()
        {
            Dictionary<string, DomainConfigurationElement> domainDict = new Dictionary<string, DomainConfigurationElement>();
            try
            {
                DomainExtractionConfig config = (DomainExtractionConfig)ConfigurationManager.GetSection("DomainExtractionConfig");

                foreach (DomainConfigurationElement domainConfig in config.DomainsConfig)
                {

                    domainDict.Add(domainConfig.Name, domainConfig);
                }
            }
            catch (Exception e)
            {
                MessageBox.Show(e.Message);
            }

            return domainDict;
        }

        private string connString;
        private void Form1_Load(object sender, EventArgs e)
        {
            // TODO: This line of code loads data into the 'crawlPagesDataSet1.FastUrlTable' table. You can move, or remove it, as needed.
            //this.fastUrlTableTableAdapter.Fill(this.crawlPagesDataSet1.FastUrlTable);
            this.GetUrlListButton.Click += GetUrlListButton_Click;
            this.dataGridView1.CellClick += RowEnterClickOfUrlList;
            domainsConfigDict = GetDomainConfigurationDict();
            this.comboBoxSelectDomain.SelectedValueChanged += new EventHandler(comboBoxSelectDomain_SelectedValueChanged);

        }
        private Dictionary<string, DomainConfigurationElement> domainsConfigDict = null;

        private void comboBoxSelectDomain_SelectedValueChanged(object sender, EventArgs e)
        {
            string domain = (string)this.comboBoxSelectDomain.SelectedItem;
            if (domainsConfigDict.ContainsKey(domain))
            {
                this.urlTemplate.Text = domainsConfigDict[domain].Pattern;
            }
            else
            {
                this.urlTemplate.Text = string.Empty;
            }
        }
        private string site;
        private void GetUrlListButton_Click(object sender, EventArgs e)
        {
            site =(string) this.comboBoxSelectDomain.SelectedItem;
            if (string.IsNullOrEmpty(site) || site.StartsWith("请选择"))
            {
                MessageBox.Show("请选择网站！");
            };
            string urlTemplate = this.urlTemplate.Text;
            string sqlString = "exec dbo.p_show 'FastUrlTable',N'select * from FastUrlTable where URL like ''{0}%'''";
            sqlString = string.Format(sqlString, "http://www."+site + urlTemplate);
            SqlCommand com = new SqlCommand();
            com.Connection = new SqlConnection(connString);
            com.CommandText = sqlString;
            com.CommandType = CommandType.Text;
            SqlDataAdapter sda = new SqlDataAdapter(com);
            DataSet ds = new DataSet();
            sda.Fill(ds);
            DataTable dt = new DataTable();
            dt.Columns.Add("网址");
            foreach (DataRow v in ds.Tables[0].Rows)
            {
                dt.Rows.Add(v.ItemArray[0].ToString().Replace(site,"$"));
            }
            this.StatusBar.Text = "共找到"+ dt.Rows.Count+ "个URL";
            this.dataGridView1.DataSource = dt;
            ContentFetcher.SetUrlList(dt, site);

        }

        private byte [] cacheContent = new byte[10*1024*1024];
        private long ccn = 0;
        private SqlConnection contentConn;
        private void RowEnterClickOfUrlList(object sender, DataGridViewCellEventArgs e)
        {

            string templeUrlstring =(string) this.dataGridView1[e.ColumnIndex, e.RowIndex].Value;
            string urlstring = templeUrlstring.Replace("$", site);
            this.urltextbox.Text = urlstring;
            //this.webBrowser1.DocumentText = "aaaaaaa";
            //this.webBrowser1.Url = new Uri( urlstring);
            string sqlString = "select ip, cachedContent, contentType from WebPages where URL = '{0}'";
            sqlString = string.Format(sqlString, urlstring);
            SqlCommand com = new SqlCommand();
            com.Connection =contentConn;
            if (contentConn.State == ConnectionState.Closed)
            {
                contentConn.Open();
            }
            else if (contentConn.State == ConnectionState.Fetching || contentConn.State == ConnectionState.Executing)
            {
                MessageBox.Show("连接忙，请稍后重试！");
                return;
            }


            com.CommandText = sqlString;
            com.CommandType = CommandType.Text;
            SqlDataReader sdr = com.ExecuteReader();
            string ip = "";
            string contentType = "";
            Encoding encoding = Encoding.UTF8;
            ccn = 0;
            if (sdr.Read())
            {
                ip =sdr.IsDBNull(0) ? string.Empty :  sdr.GetString(0);
                ccn = sdr.GetBytes(1, 0, cacheContent, 0, 10 * 1024 * 1024);
                contentType = sdr.GetString(2);
                if (contentType.ToLower().Contains("gb2312"))
                {
                    encoding = Encoding.GetEncoding("gb2312");
                }
                string content = encoding.GetString(cacheContent, 0, (int)ccn);
                this.webBrowser1.DocumentText = content;
            }
            else
            {
                this.webBrowser1.DocumentText = "没有在数据库中打到脱机记录！";
            }


            contentConn.Close();
        }

        private void NewWindowEventHandler(object sender, CancelEventArgs e)
        {
            e.Cancel = true;
        }



        private void ClassifyPages_Click(object sender, EventArgs e)
        {
            Regex regex_kxr = new Regex(@"BFD.BFD_ITEM_INFO = \{[^\}]*\};", RegexOptions.Singleline);
            Regex regex_kxr_keywords = new Regex(@"<meta name=""keywords"" content=""[^""]*");
            foreach (DataRow v in ((DataTable)(this.dataGridView1.DataSource)).Rows)
            {
                string url = v.ItemArray[0].ToString().Replace("$", site);
                if (IsUrlExistProductInformation(url, contentConn)) continue;
                string sqlString = "select ip, cachedContent, contentType from WebPages where URL = '{0}'";
                sqlString = string.Format(sqlString, url);
                var com = new SqlCommand();
                com.Connection = contentConn;
                if (contentConn.State == ConnectionState.Closed)
                {
                    contentConn.Open();
                }

                com.CommandText = sqlString;
                com.CommandType = CommandType.Text;
                SqlDataReader sdr = com.ExecuteReader();
                string ip = "";
                string contentType = "";
                Encoding encoding = Encoding.UTF8;
                ccn = 0;
                string content = string.Empty;
                
                if (sdr.Read())
                {
                    ip = sdr.IsDBNull(0)? string.Empty :  sdr.GetString(0);
                    ccn = sdr.GetBytes(1, 0, cacheContent, 0, 10 * 1024 * 1024);
                    contentType = sdr.GetString(2);
                    if (contentType.ToLower().Contains("gb2312"))
                    {
                        encoding = Encoding.GetEncoding("gb2312");
                    }
                    content = encoding.GetString(cacheContent, 0, (int)ccn);

                }
                sdr.Close();
                if (url.EndsWith(".html") && content.Length < 100)
                {
                    ContentFetcher.QueuePages(url);
                }
                Match m = regex_kxr.Match(content);
                Match keyword_m = regex_kxr_keywords.Match(content);
                Dictionary<string, string> props;
                string keywords = string.Empty;
                if (keyword_m.Success)
                {
                    keywords = keyword_m.Value.Substring(keyword_m.Value.LastIndexOf('"') + 1);
                }
                if (m.Success)
                {
                    string result = m.Value;
                    result = Regex.Replace(result, "[^:]//[^\n]*", this.EmptyMatch);
                    result = result.Substring(result.IndexOf("id:"));
                    props = JsonStylePropertyReader(result);
                    InsertIntoProductInformation(contentConn, url, keywords, props);
                }
                else
                {
                    
                }
                
                

            }
        }

        public static bool IsUrlExistProductInformation(string url, SqlConnection con )
        {
            if (con.State == ConnectionState.Closed)
            {
                con.Open();
            }

            SqlCommand getElement = new SqlCommand();
            getElement.CommandType = CommandType.Text;
            getElement.Connection = con;
            string selectStringT = "select count(*) from [ProductInfomation] where url='{0}'";
            int result = 0;

            getElement.CommandText = string.Format(selectStringT, url);
            try
            {
                result = (int)getElement.ExecuteScalar();
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
            }
            if (result > 0)
                return true;
            return false;
        }

        private bool InsertIntoProductInformation(SqlConnection con, string url, string keyword, Dictionary<string,string> props)
        {
            if (props.Count < 12) return false;
            try
            {
                var com = new SqlCommand();
                com.Connection = contentConn;
                if (contentConn.State == ConnectionState.Closed)
                {
                    contentConn.Open();
                }
                
                keyword = keyword.Substring(0, keyword.Length > 200 ? 200 : keyword.Length);
                keyword = keyword.Replace("'", "''");
                string sqlString = "insert into ProductInfomation([URL],[innerCategory],[price] ,[re_price] ,[brand],[onsale],[storeCount],[name],[description],[keywords]) values"+
                    "('{0}','{1}',{2},{3},'{4}',{5},{6},'{7}','{8}','{9}')";
                com.CommandText = string.Format(sqlString, url, props["category"].Replace("'", "''"), props["price"], props["re_price"], props["brand"].Replace("'", "''"), props["onsale"] == "true" ? 1 : 0, props["store"], props["name"].Replace("'", "''"), props["sub_name"].Replace("'", "''"), keyword);
                com.CommandType = CommandType.Text;
                int ret =  com.ExecuteNonQuery();
                if (ret == 1) return true;
                else return false;
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
                return false;
            }
        }

        private bool IsProductInfoExist(SqlConnection con, string url)
        {
            try
            {
                var com = new SqlCommand();
                com.Connection = contentConn;
                if (contentConn.State == ConnectionState.Closed)
                {
                    contentConn.Open();
                }
                string sqlString = "select count(*) from ProductInfomation where url ='{0}'";
                com.CommandText = string.Format(sqlString, url);
                com.CommandType = CommandType.Text;
                int ret =(int) com.ExecuteScalar();
                if (ret == 1) return true;
                else return false;
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
                return false;
            }
        }

        private string EmptyMatch(Match m)
        {
            if (m.Value.StartsWith(","))
                return ",";
            return string.Empty;
        }

        private Dictionary<string, string> JsonStylePropertyReader(string str)
        {
            Dictionary<string,string> result = new Dictionary<string,string>();
            StringBuilder k = new StringBuilder();
            StringBuilder v = new StringBuilder();
            bool IsQuoted = false;
            bool isK = true;
            
            for(int i = 0;i < str.Length; i++)
            {
                if (IsQuoted  )
                {
                    if (str[i] != '"' || (str[i-1]=='\\' && str[i] == '"'))
                    {
                        v.Append(str[i]);
                        continue;
                    }
                    else
                    {
                        IsQuoted = false;
                        continue;
                    }
                }
                
                if(str[i] == '{' || str[i] == '\t' || str[i] == '\r' || str[i] == '\n' || str[i] == ' ') continue;
                if (str[i] == '}' && !string.IsNullOrEmpty(k.ToString())) { result.Add(k.ToString(), v.ToString()); break; } 
                if(str[i] == ':')  {isK = false; continue;}
                if (isK) k.Append(str[i]);
                else 
                {
                    if (str[i] == '"')
                    {
                        IsQuoted = true; continue;
                    }
                    else if (str[i] == ',')
                    {
                        result.Add(k.ToString(), v.ToString());
                        k.Clear();
                        v.Clear();
                        isK = true;
                    }
                    else
                    {
                        v.Append(str[i]);
                    }
                }
                
            }

            return result;
        }

        private void urlTemplate_KeyDown(object sender, KeyEventArgs e)
        {
            if (e.KeyCode == Keys.Enter)
            {
                GetUrlListButton_Click(null, null);
            }
        }

        private void buttonReget_Click(object sender, EventArgs e)
        {
                string sqlString = "with tempUrl as (select url from WebPages where  URL  like 'http://www.360kxr.com/product/%.html') Select url from tempUrl where URL not in (select URL from ProductInfomation)";
                var com = new SqlCommand();
                com.Connection = contentConn;
                if (contentConn.State == ConnectionState.Closed)
                {
                    contentConn.Open();
                }

                com.CommandText = sqlString;
                com.CommandType = CommandType.Text;
                SqlDataReader sdr = com.ExecuteReader();
                string url = string.Empty;

                while (sdr.Read())
                {

                    url = sdr.GetString(0);
                    ContentFetcher.QueuePages(url);

                }
                sdr.Close();


        }

        public Queue<string> buffer = new Queue<string>();

        public Thread t=null;
        private void 内容析取ToolStripMenuItem_Click(object sender, EventArgs e)
        {
            
            t = new Thread(new ParameterizedThreadStart(WriteToBuffer));
            t.Start(new object());
            BufferScroll BufferForm = new BufferScroll(this);
            BufferForm.ShowDialog();
        }

        public MemoryStream ms;
        
        private void WriteToBuffer(object o)
        {
            //BufferScroll bf = (BufferScroll)o;
            int i = 0;
            while (true)
            {
                i++;
                lock (buffer)
                {
                    buffer.Enqueue(i + "____Line!");

                }
                Thread.Sleep(10);
            }
            //byte[] b = new byte[500000];
            //ms = new MemoryStream(b);
            //int i=0;
            //while (true)
            //{
            //    i++;
            //    lock (ms)
            //    {
            //        if (ms.Position > 450000)
            //        {
            //            Array.Copy(b, 300000, b, 0, ms.Position - 300000);
            //            ms.Position = ms.Position - 300000;
            //        }
            //        StreamWriter sw =new StreamWriter(ms,Encoding.Unicode);
            //            sw.Write("shit_" + i + "\n");
            //            sw.Flush();

            //    }
            //    Thread.Sleep(10);
                
            //}
        }
    }
}
